{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = []\n",
    "with open('./test.json', 'r') as f1:\n",
    "    for line in f1.readlines():\n",
    "        if line:\n",
    "            all_data.append(eval(line))\n",
    "with open('./train.json', 'r') as f2:\n",
    "    for line in f2.readlines():\n",
    "        if line:\n",
    "            all_data.append(eval(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4298\n"
     ]
    }
   ],
   "source": [
    "print(len(all_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'./food_nu.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'food' in keys and 'nutrient' in keys:\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for food in data['label']['food']:\n",
    "                for nu in data['label']['nutrient']:\n",
    "                    if '含' in data['text'] or '中的' in data['text']:\n",
    "                        f3.writelines(\n",
    "                              '含有'\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                              +','\n",
    "                              +nu\n",
    "                              +\",nutrient,\"\n",
    "                              +str(data['label']['nutrient'][nu][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                              +','\n",
    "                              +nu\n",
    "                              +\",nutrient,\"\n",
    "                              +str(data['label']['nutrient'][nu][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_buyi(text):\n",
    "    for i in ['不宜', '忌', '慎', '少食', '少吃', '不适合', '少进食', '禁', '不可', '避免', '不要', '不推荐', '少摄取']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def check_yi(text):\n",
    "    for i in ['适宜', '多吃', '多食', '适合', '有好处', '适当增加', '适量', '适用于', '多摄取', '推荐']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False  \n",
    "\n",
    "with open(r'./food_crowd.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'food' in keys and 'crowd' in keys:\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for food in data['label']['food']:\n",
    "                for crowd in data['label']['crowd']:\n",
    "                    if check_buyi(data['text']):\n",
    "                        f3.writelines('不宜,'\n",
    "                              +crowd\n",
    "                              +\",crowd,\"\n",
    "                              +str(data['label']['crowd'][crowd][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    elif check_yi(data['text']):\n",
    "                        f3.writelines('适宜,'\n",
    "                              +crowd\n",
    "                              +\",crowd,\"\n",
    "                              +str(data['label']['crowd'][crowd][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +crowd\n",
    "                              +\",crowd,\"\n",
    "                              +str(data['label']['crowd'][crowd][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1905"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_buyi(text):\n",
    "    for i in ['不宜', '忌', '慎', '少食', '少吃', '不适合', '少进食', '禁', '不可', '避免', '不要', '不推荐', '少摄取']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def check_yi(text):\n",
    "    for i in ['适宜', '多吃', '多食', '适合', '有好处', '适当增加', '适量', '适用于', '多摄取', '推荐', '预防', '改善']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False  \n",
    "\n",
    "with open(r'./disease_food.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'disease' in keys and 'food' in keys:\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for disease in data['label']['disease']:\n",
    "                for food in data['label']['food']:\n",
    "                    if check_buyi(data['text']):\n",
    "                        f3.writelines('不宜,'\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    elif check_yi(data['text']):\n",
    "                        f3.writelines('适宜,'\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +food\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_buyi(text):\n",
    "    for i in ['不宜', '忌', '慎', '少食', '少吃', '不适合', '少进食', '禁', '不可', '避免', '不要', '不推荐', '少摄取']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def check_yi(text):\n",
    "    for i in ['适宜', '多吃', '多食', '适合', '有好处', '适当增加', '适量', '适用于', '多摄取', '推荐', '预防', '改善', '防止', '消除', '有利于', '缓解', '缓解']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "buyi = []\n",
    "yi = []\n",
    "unk = []\n",
    "\n",
    "with open(r'./disease_nutrient.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'disease' in keys and 'nutrient' in keys:\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for disease in data['label']['disease']:\n",
    "                for nutrient in data['label']['nutrient']:\n",
    "                    if check_buyi(data['text']):\n",
    "                        f3.writelines('不宜,'\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +nutrient\n",
    "                              +',nutrient,'\n",
    "                              +str(data['label']['nutrient'][nutrient][0][0])\n",
    "                             )\n",
    "                    elif check_yi(data['text']):\n",
    "                        f3.writelines('适宜,'\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +nutrient\n",
    "                              +',nutrient,'\n",
    "                              +str(data['label']['nutrient'][nutrient][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +disease\n",
    "                              +\",disease,\"\n",
    "                              +str(data['label']['disease'][disease][0][0])\n",
    "                              +','\n",
    "                              +nutrient\n",
    "                              +',nutrient,'\n",
    "                              +str(data['label']['nutrient'][nutrient][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_has(text):\n",
    "    for i in ['含', '里的', '中的']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "with open(r'./food_nutrient.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'nutrient' in keys and 'food' in keys:\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for nutrient in data['label']['nutrient']:\n",
    "                for food in data['label']['food']:\n",
    "                    if check_has(data['text']):\n",
    "                        f3.writelines('含有,'\n",
    "                              +food\n",
    "                              +\",food,\"\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                              +','\n",
    "                              +nutrient\n",
    "                              +',nutrient,'\n",
    "                              +str(data['label']['nutrient'][nutrient][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +food\n",
    "                              +\",food,\"\n",
    "                              +str(data['label']['food'][food][0][0])\n",
    "                              +','\n",
    "                              +nutrient\n",
    "                              +',nutrient,'\n",
    "                              +str(data['label']['nutrient'][nutrient][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_has(text):\n",
    "    for i in ['又叫', '也叫', '亦称', '也称', '古称', '被称', '原称', '称']:\n",
    "        if i in text:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "with open(r'./food_food.txt', 'a+') as f3:\n",
    "    for data in all_data:\n",
    "        keys = list(data['label'].keys())\n",
    "        if 'food' in keys and len(data['label']['food']) >= 2 and check_has(data['text']):\n",
    "#             print(data['text'])\n",
    "            f3.writelines(data['text'])\n",
    "            f3.writelines('\\n')\n",
    "            for food1 in data['label']['food']:\n",
    "                for food2 in data['label']['food']:\n",
    "                    if (food1 == food2):\n",
    "                        continue\n",
    "                    if check_has(data['text']):\n",
    "                        f3.writelines('别名,'\n",
    "                              +food1\n",
    "                              +\",food,\"\n",
    "                              +str(data['label']['food'][food1][0][0])\n",
    "                              +','\n",
    "                              +food2\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food2][0][0])\n",
    "                             )\n",
    "                    else:\n",
    "                        f3.writelines(','\n",
    "                              +food1\n",
    "                              +\",food,\"\n",
    "                              +str(data['label']['food'][food1][0][0])\n",
    "                              +','\n",
    "                              +food2\n",
    "                              +',food,'\n",
    "                              +str(data['label']['food'][food2][0][0])\n",
    "                             )\n",
    "                    f3.writelines('\\n')\n",
    "                break\n",
    "            f3.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open(r'./data.txt', 'r') as f:\n",
    "    cur_text = ''\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        if not i:\n",
    "            cur_text = ''\n",
    "            continue\n",
    "        if cur_text == '':\n",
    "            cur_text = i\n",
    "            continue\n",
    "        cur_text = cur_text.replace(',', '，')\n",
    "        cur_text = cur_text.replace('﹐', '，')\n",
    "        i = i.replace('unkown', 'unKnown')\n",
    "        i = i.replace('unkwon', 'unKnown')\n",
    "        i = i.replace('unKown', 'unKnown')\n",
    "        data.append(cur_text + ',' + i)\n",
    "\n",
    "with open(r'./crowd_food_relations.txt', 'a+') as f:\n",
    "    for i in data:\n",
    "        f.writelines(i)\n",
    "        f.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open(r'./data2.txt', 'r') as f:\n",
    "    cur_text = ''\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        if not i:\n",
    "            cur_text = ''\n",
    "            continue\n",
    "        if cur_text == '':\n",
    "            cur_text = i\n",
    "            continue\n",
    "        cur_text = cur_text.replace(',', '，')\n",
    "        cur_text = cur_text.replace('﹐', '，')\n",
    "        i = i.replace('unkown', 'unKnown')\n",
    "        i = i.replace('unkwon', 'unKnown')\n",
    "        i = i.replace('unKown', 'unKnown')\n",
    "        data.append(cur_text + ',' + i)\n",
    "\n",
    "with open(r'./crowd_nutrient_relations.txt', 'a+') as f:\n",
    "    for i in data:\n",
    "        f.writelines(i)\n",
    "        f.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open(r'./data3.txt', 'r') as f:\n",
    "    cur_text = ''\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        if not i:\n",
    "            cur_text = ''\n",
    "            continue\n",
    "        if cur_text == '':\n",
    "            cur_text = i\n",
    "            continue\n",
    "        cur_text = cur_text.replace(',', '，')\n",
    "        cur_text = cur_text.replace('﹐', '，')\n",
    "        i = i.replace('unkown', 'unKnown')\n",
    "        i = i.replace('unKown', 'unKnown')\n",
    "        data.append(cur_text + ',' + i)\n",
    "\n",
    "with open(r'./food_nutrient_relations.txt', 'a+') as f:\n",
    "    for i in data:\n",
    "        f.writelines(i)\n",
    "        f.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open(r'./data4.txt', 'r') as f:\n",
    "    cur_text = ''\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        if not i:\n",
    "            cur_text = ''\n",
    "            continue\n",
    "        if cur_text == '':\n",
    "            cur_text = i\n",
    "            continue\n",
    "        cur_text = cur_text.replace(',', '，')\n",
    "        cur_text = cur_text.replace('﹐', '，')\n",
    "        i = i.replace('unkown', 'unKnown')\n",
    "        i = i.replace('unKown', 'unKnown')\n",
    "        data.append(cur_text + ',' + i)\n",
    "\n",
    "with open(r'./food_disease_relations.txt', 'a+') as f:\n",
    "    for i in data:\n",
    "        f.writelines(i)\n",
    "        f.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1713\n",
      "{'unKnown', '适宜', '不宜'}\n"
     ]
    }
   ],
   "source": [
    "s = set()\n",
    "count = 0\n",
    "with open(r'./food_disease_relations.txt', 'r') as f:\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        if i:\n",
    "            t = i.split(',')\n",
    "            if len(t) != 8:\n",
    "                print(i)\n",
    "            s.add(t[1])\n",
    "            if t[1] == '':\n",
    "                print(i)\n",
    "            count = count + 1\n",
    "print(count)\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(len(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5237\n",
      "111\n",
      "111\n"
     ]
    }
   ],
   "source": [
    "## split\n",
    "import random\n",
    "import math\n",
    "\n",
    "t = []\n",
    "with open(r'./checked_data/all.txt', 'r') as f:\n",
    "    for i in f.readlines():\n",
    "        i = i.strip()\n",
    "        t.append(i)\n",
    "\n",
    "random.shuffle(t)\n",
    "print(len(t))\n",
    "\n",
    "s_n = math.floor(len(t)/10 * 8)\n",
    "train = t[:s_n]\n",
    "test = t[s_n:]\n",
    "\n",
    "with open(r'./checked_data/train.csv', 'a+') as f1:\n",
    "    f1.writelines('sentence,relation,head,head_type,head_offset,tail,tail_type,tail_offset')\n",
    "    print(111)\n",
    "    f1.writelines('\\n')\n",
    "    for i in train:\n",
    "        f1.writelines(i)\n",
    "        f1.writelines('\\n')\n",
    "\n",
    "with open(r'./checked_data/test.csv', 'a+') as f2:\n",
    "    f2.writelines('sentence,relation,head,head_type,head_offset,tail,tail_type,tail_offset')\n",
    "    print(111)\n",
    "    f2.writelines('\\n')\n",
    "    for i in test:\n",
    "        f2.writelines(i)\n",
    "        f2.writelines('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
